import json

import requests as requests
from lxml import etree

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 ' \
             'Safari/537.36'
HEADERS = {'User-Agent': USER_AGENT}


def get_request(url):
    response = requests.get(url=url, headers=HEADERS)
    # print(response.text)
    return response


def get_author(text):
    html = etree.HTML(text)
    author_info = html.xpath('//*[@id="content"]/div[4]/div/div/div[1]/div[4]/p[2]//text()')
    for author_tag in author_info:
        author_tag = author_tag.strip()
        if author_tag == 'Author:' or author_tag == 'Maintainer:':
            continue
        if not author_tag:
            continue
        return author_tag


project_list = []
pypi_json = open('Include/top-pypi-packages-365-days.json', 'r', encoding='utf-8')
pypi_dict = json.loads(pypi_json.read())
for project_dict in pypi_dict['rows']:
    project_list.append(project_dict['project'])
print(len(project_list))

project_list = project_list[:50]
for project in project_list:
    pypi_url = 'https://pypi.org/project/' + project
    print(pypi_url)
    response = get_request(pypi_url)
    if response:
        author = get_author(response.text)
        print(author)
